notebook.community

Edit and run

Running simple example through EC2 using spot pricing

To work one need to have spot instances running





In [1]:



    


%%bash
. ~/.bashrc
pip install  --upgrade git+https://git@github.com/JonasWallin/linkingEC2



    


















    






Collecting git+https://git@github.com/JonasWallin/linkingEC2
  Cloning https://git@github.com/JonasWallin/linkingEC2 to /var/folders/ct/zpjk2t696k5bv03sqq9xrpd00000gn/T/pip-Dc9NwP-build
Requirement already up-to-date: boto in /Users/jonaswallin/anaconda/lib/python2.7/site-packages (from linkingEC2==0.1)
Installing collected packages: linkingEC2
  Found existing installation: linkingEC2 0.1
    Uninstalling linkingEC2-0.1:
      Successfully uninstalled linkingEC2-0.1
  Running setup.py install for linkingEC2
Successfully installed linkingEC2-0.1

















In [2]:



    


from linkingEC2 import LinkingHandler
from ConfigParser import ConfigParser

config = ConfigParser()
starfigconfig_folder = "/Users/jonaswallin/.starcluster/"
config.read(starfigconfig_folder + "config")

acess_key_id     = config.get('aws info', 'aws_access_key_id'    , 0)
aws_secret_key   = config.get('aws info', 'aws_secret_access_key', 0)
aws_region_name  = config.get('aws info', 'aws_region_name'      , 0)
my_key_loc       = config.get('key mykeyABC', 'key_location',0)


linker = LinkingHandler(aws_secret_access_key = aws_secret_key,
                        aws_access_key_id     = acess_key_id,
                        aws_region_name       = aws_region_name,
                        key_location          = my_key_loc,
                        key_name              = 'mykeyABC' )



    











In [3]:



    


linker.connect_spot_instances()



    


















    






checking if ssh into node007 works:
*
checking if ssh into node008 works:
*
checking if ssh into node009 works:
*
checking if ssh into node010 works:
*
checking if ssh into node011 works:
*
checking if ssh into node012 works:
*
collecting node007 number of processes:done
collecting node008 number of processes:done
collecting node009 number of processes:done
collecting node010 number of processes:done
collecting node011 number of processes:done
collecting node012 number of processes:done
copying ~/.ssh/mykeyABC.rsa to external ~/.ssh/id_rsa
copying files to node007 done
copying files to node008 done
copying files to node009 done
copying files to node010 done
copying files to node011 done
copying files to node012 done
disable StrictHostKeyChecking  in node007:appending to ~/nodefile in node007 done
appending to /etc/hosts in node007 done

















In [4]:



    


PACKAGES_APT = [' libatlas3-base',
                'libatlas-base-dev',
                'python-dev', 
                'openmpi-bin', 
                'libopenmpi-dev', 
                'python-numpy',
                'python-sklearn',
               'python-matplotlib',
               'git',
                'python-scipy',
               'r-base',
               'r-base-core']
PACKAGES_PIP = ['cython',
                'mpi4py',
                'simplejson',
               'rpy2']

#Adding later version of R
#http://philipp-burckhardt.com/2014/05/25/installing-r-rstudio-on-ubuntu/
command = 'sudo add-apt-repository "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/"'
linker.send_command_ssh(command = 'gpg --keyserver pgpkeys.mit.edu --recv-key 51716619E084DAB9')
linker.send_command_ssh(command = 'gpg -a --export 51716619E084DAB9 | sudo apt-key add -')
linker.send_command_ssh(command = command)



linker.apt_install(PACKAGES_APT)

#problem with memory installing scipy:
#http://naokiwatanabe.blogspot.se/2014/12/install-numpy-schipy-matplotlib-and-etc.html
linker.send_command_ssh(command = 'sudo /bin/dd if=/dev/zero of=/var/swap.1 bs=1M count=1024')
linker.send_command_ssh(command = 'sudo /sbin/mkswap /var/swap.1')
linker.send_command_ssh(command = 'sudo /sbin/swapon /var/swap.1')
linker.pip_install('-U scipy')

linker.send_command_ssh(command = 'sudo swapoff /var/swap.1')
linker.send_command_ssh(command = 'sudo sudo rm /var/swap.1')



linker.pip_install(PACKAGES_PIP)
linker.pip_install(['git+https://git@github.com/JonasWallin/BayesFlow'])
import os
os.system('say "your packages is downloaded"')



    


















    






node007, sshing in  gpg --keyserver pgpkeys.mit.edu --recv-key 51716619E084DAB9 
node007 done
node008, sshing in  gpg --keyserver pgpkeys.mit.edu --recv-key 51716619E084DAB9 
node008 done
node009, sshing in  gpg --keyserver pgpkeys.mit.edu --recv-key 51716619E084DAB9 
node009 done
node010, sshing in  gpg --keyserver pgpkeys.mit.edu --recv-key 51716619E084DAB9 
node010 done
node011, sshing in  gpg --keyserver pgpkeys.mit.edu --recv-key 51716619E084DAB9 
node011 done
node012, sshing in  gpg --keyserver pgpkeys.mit.edu --recv-key 51716619E084DAB9 
node012 done
node007, sshing in  gpg -a --export 51716619E084DAB9 | sudo apt-key add - 
node007 done
node008, sshing in  gpg -a --export 51716619E084DAB9 | sudo apt-key add - 
node008 done
node009, sshing in  gpg -a --export 51716619E084DAB9 | sudo apt-key add - 
node009 done
node010, sshing in  gpg -a --export 51716619E084DAB9 | sudo apt-key add - 
node010 done
node011, sshing in  gpg -a --export 51716619E084DAB9 | sudo apt-key add - 
node011 done
node012, sshing in  gpg -a --export 51716619E084DAB9 | sudo apt-key add - 
node012 done
node007, sshing in  sudo add-apt-repository "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/" 
node007 done
node008, sshing in  sudo add-apt-repository "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/" 
node008 done
node009, sshing in  sudo add-apt-repository "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/" 
node009 done
node010, sshing in  sudo add-apt-repository "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/" 
node010 done
node011, sshing in  sudo add-apt-repository "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/" 
node011 done
node012, sshing in  sudo add-apt-repository "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/" 
node012 done
installing the packages :  libatlas3-base libatlas-base-dev python-dev openmpi-bin libopenmpi-dev python-numpy python-sklearn python-matplotlib git python-scipy r-base r-base-core

update apt-get for node008
update apt-get for node007
update apt-get for node011
update apt-get for node009
update apt-get for node010
update apt-get for node012
node008 is done
node007 is done
node011 is done
node009 is done
node010 is done
node012 is done
upgrade apt-get for node008
upgrade apt-get for node007
upgrade apt-get for node011
upgrade apt-get for node009
upgrade apt-get for node010
upgrade apt-get for node012
node008 is done
node007 is done
node011 is done
node009 is done
node010 is done
node012 is done
node007, sshing in  sudo /bin/dd if=/dev/zero of=/var/swap.1 bs=1M count=1024 
installing package for node008
installing package for node007
installing package for node009
installing package for node010
installing package for node011
installing package for node012
node008 is done
node007 is done
node009 is done
node010 is done
node011 is done
node012 is done
node007 done
node008, sshing in  sudo /bin/dd if=/dev/zero of=/var/swap.1 bs=1M count=1024 
node008 done
node009, sshing in  sudo /bin/dd if=/dev/zero of=/var/swap.1 bs=1M count=1024 
node009 done
node010, sshing in  sudo /bin/dd if=/dev/zero of=/var/swap.1 bs=1M count=1024 
node010 done
node011, sshing in  sudo /bin/dd if=/dev/zero of=/var/swap.1 bs=1M count=1024 
node011 done
node012, sshing in  sudo /bin/dd if=/dev/zero of=/var/swap.1 bs=1M count=1024 
node012 done
node007, sshing in  sudo /sbin/mkswap /var/swap.1 
node007 done
node008, sshing in  sudo /sbin/mkswap /var/swap.1 
node008 done
node009, sshing in  sudo /sbin/mkswap /var/swap.1 
node009 done
node010, sshing in  sudo /sbin/mkswap /var/swap.1 
node010 done
node011, sshing in  sudo /sbin/mkswap /var/swap.1 
node011 done
node012, sshing in  sudo /sbin/mkswap /var/swap.1 
node012 done
node007, sshing in  sudo /sbin/swapon /var/swap.1 
node007 done
node008, sshing in  sudo /sbin/swapon /var/swap.1 
node008 done
node009, sshing in  sudo /sbin/swapon /var/swap.1 
node009 done
node010, sshing in  sudo /sbin/swapon /var/swap.1 
node010 done
node011, sshing in  sudo /sbin/swapon /var/swap.1 
node011 done
node012, sshing in  sudo /sbin/swapon /var/swap.1 
node012 done
installing the packages : python-pip

installing the packages : -U scipy

installing package for node007
installing package for node009
installing package for node010
installing package for node011
installing package for node008
installing package for node012
node007 is done
node009 is done
node010 is done
node011 is done
node008 is done
node012 is done
node007, sshing in  sudo swapoff /var/swap.1 
installing package for node009
installing package for node008
installing package for node007
installing package for node010
installing package for node011
installing package for node012
node009 is done
node008 is done
node007 is done
node010 is done
node011 is done
node012 is done
node007 done
node008, sshing in  sudo swapoff /var/swap.1 
node008 done
node009, sshing in  sudo swapoff /var/swap.1 
node009 done
node010, sshing in  sudo swapoff /var/swap.1 
node010 done
node011, sshing in  sudo swapoff /var/swap.1 
node011 done
node012, sshing in  sudo swapoff /var/swap.1 
node012 done
node007, sshing in  sudo sudo rm /var/swap.1 
node007 done
node008, sshing in  sudo sudo rm /var/swap.1 
node008 done
node009, sshing in  sudo sudo rm /var/swap.1 
node009 done
node010, sshing in  sudo sudo rm /var/swap.1 
node010 done
node011, sshing in  sudo sudo rm /var/swap.1 
node011 done
node012, sshing in  sudo sudo rm /var/swap.1 
node012 done
installing the packages : cython mpi4py simplejson rpy2

installing the packages : git+https://git@github.com/JonasWallin/BayesFlow

installing package for node009
installing package for node008
installing package for node007
installing package for node010
installing package for node011
installing package for node012
node009 is done
node008 is done
node007 is done
node010 is done
node011 is done
node012 is done
installing package for node008
installing package for node007
installing package for node009
installing package for node010
installing package for node011
installing package for node012
node008 is done
node007 is done
node009 is done
node010 is done
node011 is done
node012 is done










    
Out[4]:








0

















In [5]:



    


linker.send_command_ssh( command = 'wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/covs_.npy')
linker.send_command_ssh( command = 'wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/means_.npy')
linker.send_command_ssh( command = 'wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/weights_.npy')
linker.send_command_ssh( command = 'wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_util.py')
linker.send_command_ssh( command = 'wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_simulatedata.py')
linker.send_command_ssh( command = 'wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_estimate_largerdata1_mpi.py')



    


















    






node007, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/covs_.npy 
node007 done
node008, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/covs_.npy 
node008 done
node009, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/covs_.npy 
node009 done
node010, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/covs_.npy 
node010 done
node011, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/covs_.npy 
node011 done
node012, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/covs_.npy 
node012 done
node007, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/means_.npy 
node007 done
node008, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/means_.npy 
node008 done
node009, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/means_.npy 
node009 done
node010, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/means_.npy 
node010 done
node011, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/means_.npy 
node011 done
node012, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/means_.npy 
node012 done
node007, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/weights_.npy 
node007 done
node008, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/weights_.npy 
node008 done
node009, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/weights_.npy 
node009 done
node010, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/weights_.npy 
node010 done
node011, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/weights_.npy 
node011 done
node012, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/weights_.npy 
node012 done
node007, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_util.py 
node007 done
node008, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_util.py 
node008 done
node009, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_util.py 
node009 done
node010, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_util.py 
node010 done
node011, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_util.py 
node011 done
node012, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_util.py 
node012 done
node007, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_simulatedata.py 
node007 done
node008, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_simulatedata.py 
node008 done
node009, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_simulatedata.py 
node009 done
node010, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_simulatedata.py 
node010 done
node011, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_simulatedata.py 
node011 done
node012, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_simulatedata.py 
node012 done
node007, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_estimate_largerdata1_mpi.py 
node007 done
node008, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_estimate_largerdata1_mpi.py 
node008 done
node009, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_estimate_largerdata1_mpi.py 
node009 done
node010, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_estimate_largerdata1_mpi.py 
node010 done
node011, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_estimate_largerdata1_mpi.py 
node011 done
node012, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_estimate_largerdata1_mpi.py 
node012 done










    
Out[5]:








0

















In [12]:



    


import numpy as np
tot_process = np.sum([node['n_process'] for node in linker.nodes])
command = 'mpirun -hostfile nodefile -n %d python article_estimate_largerdata1_mpi.py'%(tot_process)
linker.send_command_ssh(nodes = 0, command = command)
linker.copy_files_from_node('simulation_result.npy')
linker.copy_files_from_node('mus_sim.npy')
linker.copy_files_from_node('sim_data.npy')



    


















    






node007, sshing in  mpirun -hostfile nodefile -n 192 python article_estimate_largerdata1_mpi.py 
node007 done
copying simulation_result.npy to local computer .
copying files to node007 done
copying mus_sim.npy to local computer .
copying files to node007 done
copying sim_data.npy to local computer .
copying files to node007 done

















In [10]:



    


linker.copy_files_to_nodes( file_name = '../examples/article1/article_estimate_largerdata1_mpi.py', destination='~/' )
linker.copy_files_to_nodes( file_name = '../examples/article1/article_util.py', destination='~/' )



    


















    






copying ../examples/article1/article_estimate_largerdata1_mpi.py to external ~/
copying files to node009 done
copying files to node010 done
copying ../examples/article1/article_util.py to external ~/
copying files to node009 done
copying files to node010 done

















In [10]:



    


print("ssh  -i {keyloc} -o 'StrictHostKeyChecking no'  ubuntu@{hostname}".format(
		keyloc = linker.my_key_location,
		hostname =  linker.nodes[0]['public_dns']))



    


















    






ssh  -i ~/.ssh/mykeyABC.rsa -o 'StrictHostKeyChecking no'  ubuntu@ec2-54-173-159-50.compute-1.amazonaws.com

















In [7]:



    


linker.send_command_ssh( command = 'rm ~/article_simulatedata.py')



    


















    






node007, sshing in  rm ~/article_simulatedata.py 
node007 done
node008, sshing in  rm ~/article_simulatedata.py 
node008 done
node009, sshing in  rm ~/article_simulatedata.py 
node009 done
node010, sshing in  rm ~/article_simulatedata.py 
node010 done
node011, sshing in  rm ~/article_simulatedata.py 
node011 done
node012, sshing in  rm ~/article_simulatedata.py 
node012 done










    
Out[7]:








0

















In [8]:



    


linker.send_command_ssh( command = 'wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_simulatedata.py')



    


















    






node007, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_simulatedata.py 
node007 done
node008, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_simulatedata.py 
node008 done
node009, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_simulatedata.py 
node009 done
node010, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_simulatedata.py 
node010 done
node011, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_simulatedata.py 
node011 done
node012, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_simulatedata.py 
node012 done










    
Out[8]:








0

















In [ ]:

Content source: JonasWallin/BayesFlow

Similar notebooks:

notebook.community | gallery | about